<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - cpu_dlib.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_DNN_CPU_cPP_
<font color='#0000FF'>#define</font> DLIB_DNN_CPU_cPP_

<font color='#009900'>// This file contains CPU implementations of the GPU based functions in cuda_dlib.h
</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='cpu_dlib.h.html'>cpu_dlib.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tensor_tools.h.html'>tensor_tools.h</a>"

<font color='#0000FF'>namespace</font> dlib
<b>{</b>
    <font color='#0000FF'>namespace</font> cpu 
    <b>{</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='multiply'></a>multiply</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>bool</u></font> add_to,
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src1,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src2
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> MD <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>MD<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font face='Lucida Console'>(</font>src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>MD<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font face='Lucida Console'>(</font>src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>MD<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> max_size <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                        d[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i]<font color='#5555FF'>*</font>s2[i];
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                        d[i] <font color='#5555FF'>=</font> s1[i]<font color='#5555FF'>*</font>s2[i];
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>add_to<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                        d[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> max_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    d[i<font color='#5555FF'>%</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i<font color='#5555FF'>%</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>]<font color='#5555FF'>*</font>s2[i<font color='#5555FF'>%</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> max_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                        d[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i<font color='#5555FF'>%</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>]<font color='#5555FF'>*</font>s2[i<font color='#5555FF'>%</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> max_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                        d[i] <font color='#5555FF'>=</font> s1[i<font color='#5555FF'>%</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>]<font color='#5555FF'>*</font>s2[i<font color='#5555FF'>%</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='multiply_conv'></a>multiply_conv</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>bool</u></font> add_to,
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src1,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src2
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                                <b>{</b>
                                    <font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s1<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s2[k];
                                <b>}</b>
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                                <b>{</b>
                                    <font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s1<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s2[k];
                                <b>}</b>
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>src1,src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>add_to<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                        d[k] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>

                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                            <b>{</b>
                                d[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s1<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s2<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font>;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='add'></a>add</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> beta,
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'><u>float</u></font> alpha,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                  <font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>src, dest<font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
                  <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
                  <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
                  <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  <font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>src,dest<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font> , 
                    "<font color='#CC0000'>\n\t dest.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t dest.k():           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t dest.nr():          </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t dest.nc():          </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t src.num_samples():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t src.k():            </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t src.nr():           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t src.nc():           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font face='Lucida Console'>)</font>;


            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>beta <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> alpha <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                dest <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#0000FF'>return</font>;
            <b>}</b>

            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sn <font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:n;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sk <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:k;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sr <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:r;
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sc <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:c;

                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>sn<font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sk<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sr<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sc;
                            <font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> beta<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>d<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> alpha<font color='#5555FF'>*</font>s[s_idx];
                            <font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='add'></a>add</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src1,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src2
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Do the simple and fast version if everything has the same dimensions
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest, src1<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest, src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    d[i] <font color='#5555FF'>=</font> s1[i] <font color='#5555FF'>+</font> s2[i];
                <font color='#0000FF'>return</font>;
            <b>}</b>

            <font color='#009900'>// Otherwise, do the more complex version with bounds checking.
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> v1 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                            <font color='#0000FF'><u>float</u></font> v2 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

                            <font color='#009900'>// if this index is inside src1
</font>                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                                k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                                r <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                                c <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
                                v1 <font color='#5555FF'>=</font> s1[s_idx];
                            <b>}</b>

                            <font color='#009900'>// if this index is inside src2
</font>                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'>&lt;</font> src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                                k <font color='#5555FF'>&lt;</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                                r <font color='#5555FF'>&lt;</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                                c <font color='#5555FF'>&lt;</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
                                v2 <font color='#5555FF'>=</font> s2[s_idx];
                            <b>}</b>

                            <font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> v1 <font color='#5555FF'>+</font> v2;
                            <font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='assign_bias_gradient'></a>assign_bias_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                  grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                out[i] <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>in<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>1</font>; j <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>in<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='assign_conv_bias_gradient'></a>assign_conv_bias_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                  grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                  <font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad,gradient_input<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font>
                  <font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                g[k] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        <b>{</b>
                            g[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>gi<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s[i] <font color='#5555FF'>+</font> B;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src1,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src2,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> C
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s1[i] <font color='#5555FF'>+</font> B<font color='#5555FF'>*</font>s2[i] <font color='#5555FF'>+</font> C;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src1,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src2,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src3,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> C,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> D
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s3 <font color='#5555FF'>=</font> src3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s1[i] <font color='#5555FF'>+</font> B<font color='#5555FF'>*</font>s2[i] <font color='#5555FF'>+</font> C<font color='#5555FF'>*</font>s3[i] <font color='#5555FF'>+</font> D;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='affine_transform_range'></a>affine_transform_range</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>size_t</u></font> begin,
            <font color='#0000FF'><u>size_t</u></font> end,
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src1,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src2,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src3,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> C
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>begin <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> end <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> end <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s3 <font color='#5555FF'>=</font> src3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> begin; i <font color='#5555FF'>&lt;</font> end; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s1[i] <font color='#5555FF'>+</font> B<font color='#5555FF'>*</font>s2[i] <font color='#5555FF'>+</font> C<font color='#5555FF'>*</font>s3[i];
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> A,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> B
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                  <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
                  <font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                  A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font>B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> a <font color='#5555FF'>=</font> A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> a[j]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b[j];
                        d<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                        s<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    d[i] <font color='#5555FF'>=</font> a[i]<font color='#5555FF'>*</font>s[i] <font color='#5555FF'>+</font> b[i];
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='affine_transform_conv'></a>affine_transform_conv</b><font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> A,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> B
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>A,B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> a <font color='#5555FF'>=</font> A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> a[k]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b[k];
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='compute_adam_update'></a>compute_adam_update</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>size_t</u></font> begin,
            <font color='#0000FF'><u>size_t</u></font> end,
            tensor<font color='#5555FF'>&amp;</font> s,
            tensor<font color='#5555FF'>&amp;</font> m,
            tensor<font color='#5555FF'>&amp;</font> v,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> t,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> learning_rate,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> weight_decay,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> momentum1,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> momentum2,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> params,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> m.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> v.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> params_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>begin <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> end <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> end <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha <font color='#5555FF'>=</font> learning_rate<font color='#5555FF'>*</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>momentum2,t<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>momentum1, t<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loop is equivalent to doing this:
</font>            <font color='#009900'>//   m = momentum1*m + (1-momentum1)    *   (weight_decay*params + params_grad);
</font>            <font color='#009900'>//   v = momentum2*v + (1-momentum2)*squared(weight_decay*params + params_grad);
</font>            <font color='#009900'>//   s = -alpha*m/(sqrt(v) + eps);
</font>            <font color='#0000FF'>auto</font> pm <font color='#5555FF'>=</font> m.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> pv <font color='#5555FF'>=</font> v.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> ps <font color='#5555FF'>=</font> s.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> pparams <font color='#5555FF'>=</font> params.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> ppgrad <font color='#5555FF'>=</font> params_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> begin; i <font color='#5555FF'>&lt;</font> end; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'><u>float</u></font> g <font color='#5555FF'>=</font> weight_decay<font color='#5555FF'>*</font>pparams[i] <font color='#5555FF'>+</font> ppgrad[i];
                pm[i] <font color='#5555FF'>=</font> momentum1<font color='#5555FF'>*</font>pm[i] <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>momentum1<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>g;
                pv[i] <font color='#5555FF'>=</font> momentum2<font color='#5555FF'>*</font>pv[i] <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>momentum2<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>g<font color='#5555FF'>*</font>g;
                ps[i] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>alpha<font color='#5555FF'>*</font>pm[i]<font color='#5555FF'>/</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>pv[i]<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_inference'></a>batch_normalize_inference</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
            resizable_tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gamma, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> beta,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> running_means,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> running_variances
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, beta<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                eps <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, 
                "<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.k():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>
                "<font color='#CC0000'>\neps:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps 
            <font face='Lucida Console'>)</font>;
            dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> m <font color='#5555FF'>=</font> running_means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> v <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> g[k]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s <font color='#5555FF'>-</font> m[k]<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>v[k]<font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b[k];
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>s;
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize'></a>batch_normalize</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
            resizable_tensor<font color='#5555FF'>&amp;</font> dest,
            resizable_tensor<font color='#5555FF'>&amp;</font> means,
            resizable_tensor<font color='#5555FF'>&amp;</font> invstds,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> averaging_factor,
            resizable_tensor<font color='#5555FF'>&amp;</font> running_means,
            resizable_tensor<font color='#5555FF'>&amp;</font> running_variances,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gamma, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> beta 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> averaging_factor <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> averaging_factor <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>averaging_factor: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> averaging_factor<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_means,means<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_variances,invstds<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                eps <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, 
                "<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.k():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>
                "<font color='#CC0000'>\neps:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps 
            <font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            means.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            invstds.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// first compute means and invstds
</font>            means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            invstds <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// compute means, and sum of squares
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'><u>float</u></font> val <font color='#5555FF'>=</font> p_src[n<font color='#5555FF'>*</font>num<font color='#5555FF'>+</font>i];
                    p_means[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> val;
                    p_invstds[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> val<font color='#5555FF'>*</font>val;
                <b>}</b>
            <b>}</b>
            means <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            invstds <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// copy data back to host
</font>            invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// compute variances 
</font>            running_variances.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> rvar <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// This scale makes the running variances unbiased.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> actual_var <font color='#5555FF'>=</font> p_invstds[i] <font color='#5555FF'>-</font> p_means[i]<font color='#5555FF'>*</font>p_means[i];
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                    rvar[i] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>actual_var;
                <font color='#0000FF'>else</font>
                    rvar[i] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>rvar[i] <font color='#5555FF'>+</font> scale<font color='#5555FF'>*</font>averaging_factor<font color='#5555FF'>*</font>actual_var;

                p_invstds[i] <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>actual_var <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
            <b>}</b>

            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_dest <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[i];
                    <font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_dest<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_gamma[i] <font color='#5555FF'>+</font> p_beta[i];
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_dest;
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// now keep track of the running means 
</font>            running_means.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                running_means <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> averaging_factor<font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                running_means <font color='#5555FF'>=</font> means;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_gradient'></a>batch_normalize_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> means,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> invstds,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gamma,
            tensor<font color='#5555FF'>&amp;</font> src_grad,
            tensor<font color='#5555FF'>&amp;</font> gamma_grad, 
            tensor<font color='#5555FF'>&amp;</font> beta_grad 
        <font face='Lucida Console'>)</font>
        <b>{</b>

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>means.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>invstds.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>beta_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>eps <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>auto</font> p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma_grad <font color='#5555FF'>=</font> gamma_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta_grad <font color='#5555FF'>=</font> beta_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            resizable_tensor dvars, dmeans;
            dvars.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
            dmeans.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
            dvars <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            dmeans <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dvars <font color='#5555FF'>=</font> dvars.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dmeans <font color='#5555FF'>=</font> dmeans.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x_hat <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[i];
                    p_beta_grad[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad;
                    p_gamma_grad[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_grad<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>x_hat;

                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[i];

                    p_dvars[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font><font color='#5555FF'>*</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>p_invstds[i], <font color='#979000'>3.0f</font><font face='Lucida Console'>)</font>;

                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                <b>}</b>
            <b>}</b>

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invnum <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[i];

                    p_dmeans[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font color='#5555FF'>-</font>p_invstds[i] <font color='#5555FF'>+</font> p_dvars[i] <font color='#5555FF'>*</font> <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum;

                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                <b>}</b>
            <b>}</b>
            p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_src_grad <font color='#5555FF'>=</font> src_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[i];

                    <font color='#5555FF'>*</font>p_src_grad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font>p_invstds[i] <font color='#5555FF'>+</font> 
                        p_dvars[i] <font color='#5555FF'>*</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum <font color='#5555FF'>+</font> 
                        p_dmeans[i]<font color='#5555FF'>*</font>invnum;


                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src_grad;
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_conv_inference'></a>batch_normalize_conv_inference</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
            resizable_tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gamma, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> beta,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> running_means,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> running_variances
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, beta<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                eps <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, 
                "<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.k():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_means.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_means.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nrunning_variances.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> running_variances.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>
                "<font color='#CC0000'>\neps:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps 
            <font face='Lucida Console'>)</font>;
            dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> m <font color='#5555FF'>=</font> running_means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> v <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invstd <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>v[k] <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> g[k]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s <font color='#5555FF'>-</font> m[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invstd <font color='#5555FF'>+</font> b[k];
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>s;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_conv'></a>batch_normalize_conv</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
            resizable_tensor<font color='#5555FF'>&amp;</font> dest,
            resizable_tensor<font color='#5555FF'>&amp;</font> means,
            resizable_tensor<font color='#5555FF'>&amp;</font> invstds,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> averaging_factor,
            resizable_tensor<font color='#5555FF'>&amp;</font> running_means,
            resizable_tensor<font color='#5555FF'>&amp;</font> running_variances,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gamma, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> beta 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> averaging_factor <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> averaging_factor <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>averaging_factor: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> averaging_factor<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_means,means<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_variances,invstds<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
                src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                eps <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, 
                "<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.k():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nbeta.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.k():   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nr():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> 
                "<font color='#CC0000'>\nsrc.nc():  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>
                "<font color='#CC0000'>\neps:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps 
            <font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            means.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            invstds.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// first compute means and invstds
</font>            means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            invstds <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// compute means, and sum of squares
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    <b>{</b>
                        p_means[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_src;
                        p_invstds[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src<font face='Lucida Console'>)</font>;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            means <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num;
            invstds <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num;
            <font color='#009900'>// copy data back to host
</font>            invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// compute variances 
</font>            running_variances.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> rvar <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// This scale makes the running variances unbiased.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num<font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'><u>float</u></font> actual_var <font color='#5555FF'>=</font> p_invstds[k] <font color='#5555FF'>-</font> p_means[k]<font color='#5555FF'>*</font>p_means[k];
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                    rvar[k] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>actual_var;
                <font color='#0000FF'>else</font>
                    rvar[k] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>rvar[k] <font color='#5555FF'>+</font> scale<font color='#5555FF'>*</font>averaging_factor<font color='#5555FF'>*</font>actual_var;

                p_invstds[k] <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>actual_var <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
            <b>}</b>

            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_dest <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[k];
                        <font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_dest<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_gamma[k] <font color='#5555FF'>+</font> p_beta[k];
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_dest;
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// now keep track of the running means 
</font>            running_means.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                running_means <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> averaging_factor<font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                running_means <font color='#5555FF'>=</font> means;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_conv_gradient'></a>batch_normalize_conv_gradient</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> means,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> invstds,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gamma,
            tensor<font color='#5555FF'>&amp;</font> src_grad,
            tensor<font color='#5555FF'>&amp;</font> gamma_grad, 
            tensor<font color='#5555FF'>&amp;</font> beta_grad 
        <font face='Lucida Console'>)</font>
        <b>{</b>

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>means.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>invstds.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>beta_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>eps <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>auto</font> p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma_grad <font color='#5555FF'>=</font> gamma_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta_grad <font color='#5555FF'>=</font> beta_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;   
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            resizable_tensor dvars, dmeans;
            dvars.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
            dmeans.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
            dvars <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            dmeans <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dvars <font color='#5555FF'>=</font> dvars.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dmeans <font color='#5555FF'>=</font> dmeans.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invstd_pow <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>0.5</font><font color='#5555FF'>*</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>p_invstds[k], <font color='#979000'>3.0f</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x_hat <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[k];
                        p_beta_grad[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad;
                        p_gamma_grad[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_grad<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>x_hat;

                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[k];

                        p_dvars[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invstd_pow;

                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invnum <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[k];

                        p_dmeans[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font>dx<font color='#5555FF'>*</font>p_invstds[k] <font color='#5555FF'>+</font> p_dvars[k] <font color='#5555FF'>*</font> <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum;

                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> p_src_grad <font color='#5555FF'>=</font> src_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[k];

                        <font color='#5555FF'>*</font>p_src_grad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font>p_invstds[k] <font color='#5555FF'>+</font> 
                            p_dvars[k]<font color='#5555FF'>*</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum <font color='#5555FF'>+</font> 
                            p_dmeans[k]<font color='#5555FF'>*</font>invnum;


                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src_grad;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='threshold'></a>threshold</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> data,
            <font color='#0000FF'><u>float</u></font> thresh
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> data.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> data.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> d[i]<font color='#5555FF'>&gt;</font>thresh ? <font color='#979000'>1</font>:<font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='dot'></a>dot</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> a,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> b,
            tensor<font color='#5555FF'>&amp;</font> result,
            <font color='#0000FF'><u>size_t</u></font> idx
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>a.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>idx <font color='#5555FF'>&lt;</font> result.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> aa <font color='#5555FF'>=</font> a.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> bb <font color='#5555FF'>=</font> b.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> r <font color='#5555FF'>=</font> result.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> a.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                r[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> aa[i]<font color='#5555FF'>*</font>bb[i];
        <b>}</b>

    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>    <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='softmax'></a>softmax</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// Note that we subtract out the max values in each channel before applying
</font>            <font color='#009900'>// exp() to avoid numeric overflow in the subsequent computations.  Doing this
</font>            <font color='#009900'>// doesn't change the resulting output, it just makes it more numerically
</font>            <font color='#009900'>// stable.
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> ss <font color='#5555FF'>=</font> s <font color='#5555FF'>+</font> num<font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
                <font color='#0000FF'>auto</font> dd <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> num<font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'><u>float</u></font> max_val <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>std::numeric_limits<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>infinity</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                        max_val <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>max_val, ss[k<font color='#5555FF'>*</font>num]<font face='Lucida Console'>)</font>;

                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                        dd[k<font color='#5555FF'>*</font>num] <font color='#5555FF'>=</font> std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>ss[k<font color='#5555FF'>*</font>num]<font color='#5555FF'>-</font>max_val<font face='Lucida Console'>)</font>;

                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ss;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>dd;
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// Now normalize each channel so they sum to 1.
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> dd <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> num<font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> ddd <font color='#5555FF'>=</font> dd<font color='#5555FF'>+</font>r<font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c;

                        <font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                            temp <font color='#5555FF'>+</font><font color='#5555FF'>=</font> ddd[k<font color='#5555FF'>*</font>num];
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                            ddd[k<font color='#5555FF'>*</font>num] <font color='#5555FF'>/</font><font color='#5555FF'>=</font> temp;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='softmax_gradient'></a>softmax_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d2 <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> num<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g2 <font color='#5555FF'>=</font> g <font color='#5555FF'>+</font> num<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in2 <font color='#5555FF'>=</font> in <font color='#5555FF'>+</font> num<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d3 <font color='#5555FF'>=</font> d2<font color='#5555FF'>+</font>r<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c;
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g3 <font color='#5555FF'>=</font> g2<font color='#5555FF'>+</font>r<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c;
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in3 <font color='#5555FF'>=</font> in2<font color='#5555FF'>+</font>r<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c;

                        <font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                            temp <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font>d3[k<font color='#5555FF'>*</font>num]<font color='#5555FF'>*</font>in3[k<font color='#5555FF'>*</font>num];
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                                g3[k<font color='#5555FF'>*</font>num] <font color='#5555FF'>=</font> d3[k<font color='#5555FF'>*</font>num]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>temp<font color='#5555FF'>+</font>in3[k<font color='#5555FF'>*</font>num]<font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                                g3[k<font color='#5555FF'>*</font>num] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> d3[k<font color='#5555FF'>*</font>num]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>temp<font color='#5555FF'>+</font>in3[k<font color='#5555FF'>*</font>num]<font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='sigmoid'></a>sigmoid</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>+</font>std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>s[i]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='sigmoid_gradient'></a>sigmoid_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    g[i] <font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font>d[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    g[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font>d[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='relu'></a>relu</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src
        <font face='Lucida Console'>)</font>
        <b>{</b>
            dest <font color='#5555FF'>=</font> <font color='#BB00BB'>lowerbound</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='relu_gradient'></a>relu_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> in <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>in[i] <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                        out[i] <font color='#5555FF'>=</font> gi[i];
                    <font color='#0000FF'>else</font>
                        out[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>in[i] <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                        out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i];
                <b>}</b>
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='prelu'></a>prelu</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> param
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> p <font color='#5555FF'>=</font> param.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s[i] <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    d[i] <font color='#5555FF'>=</font> s[i];
                <font color='#0000FF'>else</font>
                    d[i] <font color='#5555FF'>=</font> p<font color='#5555FF'>*</font>s[i];
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='prelu_gradient'></a>prelu_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> param,
            tensor<font color='#5555FF'>&amp;</font> params_grad 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> p <font color='#5555FF'>=</font> param.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> pgrad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s[i] <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i];
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> p<font color='#5555FF'>*</font>gi[i];
                    pgrad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i]<font color='#5555FF'>*</font>s[i];
                <b>}</b>
            <b>}</b>
            params_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>] <font color='#5555FF'>=</font> pgrad;
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='tanh'></a>tanh</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                d[i] <font color='#5555FF'>=</font> std::<font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='tanh_gradient'></a>tanh_gradient</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> grad,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    g[i] <font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font color='#5555FF'>*</font>d[i]<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    g[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font color='#5555FF'>*</font>d[i]<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        pooling::<b><a name='pooling'></a>pooling</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> : window_height<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,window_width<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,stride_y<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,stride_x<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,padding_y<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,padding_x<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,do_max_pooling<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> pooling::
        <b><a name='clear'></a>clear</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            window_height <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            window_width <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            stride_y <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            stride_x <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            padding_y <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            padding_x <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> pooling::
        <b><a name='setup_max_pooling'></a>setup_max_pooling</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>int</u></font> window_height_,
            <font color='#0000FF'><u>int</u></font> window_width_,
            <font color='#0000FF'><u>int</u></font> stride_y_,
            <font color='#0000FF'><u>int</u></font> stride_x_,
            <font color='#0000FF'><u>int</u></font> padding_y_,
            <font color='#0000FF'><u>int</u></font> padding_x_
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_x_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_y_ <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_y_ <font color='#5555FF'>&lt;</font> window_height_<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_x_ <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_x_ <font color='#5555FF'>&lt;</font> window_width_<font face='Lucida Console'>)</font>;

            window_height <font color='#5555FF'>=</font> window_height_;
            window_width <font color='#5555FF'>=</font> window_width_;
            stride_y <font color='#5555FF'>=</font> stride_y_;
            stride_x <font color='#5555FF'>=</font> stride_x_;
            padding_y <font color='#5555FF'>=</font> padding_y_;
            padding_x <font color='#5555FF'>=</font> padding_x_;
            do_max_pooling <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> pooling::
        <b><a name='setup_avg_pooling'></a>setup_avg_pooling</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>int</u></font> window_height_,
            <font color='#0000FF'><u>int</u></font> window_width_,
            <font color='#0000FF'><u>int</u></font> stride_y_,
            <font color='#0000FF'><u>int</u></font> stride_x_,
            <font color='#0000FF'><u>int</u></font> padding_y_,
            <font color='#0000FF'><u>int</u></font> padding_x_
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_x_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_y_ <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_y_ <font color='#5555FF'>&lt;</font> window_height_<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_x_ <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_x_ <font color='#5555FF'>&lt;</font> window_width_<font face='Lucida Console'>)</font>;

            window_height <font color='#5555FF'>=</font> window_height_;
            window_width <font color='#5555FF'>=</font> window_width_;
            stride_y <font color='#5555FF'>=</font> stride_y_;
            stride_x <font color='#5555FF'>=</font> stride_x_;
            padding_y <font color='#5555FF'>=</font> padding_y_;
            padding_x <font color='#5555FF'>=</font> padding_x_;
            do_max_pooling <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> pooling::
        <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            resizable_tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_y <font color='#5555FF'>&lt;</font> window_height<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_x <font color='#5555FF'>&lt;</font> window_width<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width  <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x,
                "<font color='#CC0000'>Pooling windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y,
                "<font color='#CC0000'>Pooling windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>
                 src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>window_height<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y,
                 <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>window_width<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x
                <font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                dest <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#0000FF'>return</font>;
            <b>}</b>


            <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>does_max_pooling</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> dimg <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                    r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                    window_width,
                                    window_height<font face='Lucida Console'>)</font>;
                                dimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font>simg,win<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> dimg <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                    r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                    window_width,
                                    window_height<font face='Lucida Console'>)</font>;
                                dimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font>simg,win<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>

        <b>}</b>

        <font color='#0000FF'><u>void</u></font> pooling::<b><a name='get_gradient'></a>get_gradient</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            tensor<font color='#5555FF'>&amp;</font> grad 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input,dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>src,grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font>;
            <b>}</b>


            <font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>does_max_pooling</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> gimg <font color='#5555FF'>=</font> g <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> giimg <font color='#5555FF'>=</font> gi <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> imgbox <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>simg<font face='Lucida Console'>)</font>;

                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                    r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                    window_width,
                                    window_height<font face='Lucida Console'>)</font>.<font color='#BB00BB'>intersect</font><font face='Lucida Console'>(</font>imgbox<font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>auto</font> p <font color='#5555FF'>=</font> <font color='#BB00BB'>max_point</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm</font><font face='Lucida Console'>(</font>simg,win<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>win.<font color='#BB00BB'>tl_corner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                                gimg[p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> giimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c];
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> gimg <font color='#5555FF'>=</font> g <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> giimg <font color='#5555FF'>=</font> gi <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>auto</font> imgbox <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>simg<font face='Lucida Console'>)</font>;

                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                    r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                    window_width,
                                    window_height<font face='Lucida Console'>)</font>.<font color='#BB00BB'>intersect</font><font face='Lucida Console'>(</font>imgbox<font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> delta <font color='#5555FF'>=</font> giimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c]<font color='#5555FF'>/</font>win.<font color='#BB00BB'>area</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> win.<font color='#BB00BB'>top</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; y <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> win.<font color='#BB00BB'>bottom</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
                                <b>{</b>
                                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> x <font color='#5555FF'>=</font> win.<font color='#BB00BB'>left</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; x <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> win.<font color='#BB00BB'>right</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>x<font face='Lucida Console'>)</font>
                                    <b>{</b>
                                        gimg[y<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>x] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> delta;
                                    <b>}</b>
                                <b>}</b>
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>

        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='img2col'></a>img2col</b><font face='Lucida Console'>(</font>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> data,
            <font color='#0000FF'><u>long</u></font> n,
            <font color='#0000FF'><u>long</u></font> filter_nr,
            <font color='#0000FF'><u>long</u></font> filter_nc,
            <font color='#0000FF'><u>long</u></font> stride_y,
            <font color='#0000FF'><u>long</u></font> stride_x,
            <font color='#0000FF'><u>long</u></font> padding_y,
            <font color='#0000FF'><u>long</u></font> padding_x
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> data.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
            <font color='#0000FF'>const</font> rectangle boundary <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> out_nr <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>filter_nr<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> out_nc <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>filter_nc<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x;

            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>out_nr<font color='#5555FF'>*</font>out_nc, 
                            data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>filter_nr<font color='#5555FF'>*</font>filter_nc<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> t <font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font><font color='#BB00BB'>output</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// now fill in the Toeplitz output matrix for the n-th sample in data.  
</font>            <font color='#0000FF'><u>size_t</u></font> cnt <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_r <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_y<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nr<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_c <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_x<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nc<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_y; r <font color='#5555FF'>&lt;</font> max_r; r<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_y<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_x; c <font color='#5555FF'>&lt;</font> max_c; c<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_x<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> <font color='#979000'>0</font>; y <font color='#5555FF'>&lt;</font> filter_nr; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> x <font color='#5555FF'>=</font> <font color='#979000'>0</font>; x <font color='#5555FF'>&lt;</font> filter_nc; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>x<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>cnt <font color='#5555FF'>&lt;</font> output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                <font color='#0000FF'><u>long</u></font> xx <font color='#5555FF'>=</font> c<font color='#5555FF'>+</font>x;
                                <font color='#0000FF'><u>long</u></font> yy <font color='#5555FF'>=</font> r<font color='#5555FF'>+</font>y;
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>boundary.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font>xx,yy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                                    <font color='#5555FF'>*</font>t <font color='#5555FF'>=</font> d[<font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> yy<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> xx];
                                <font color='#0000FF'>else</font>
                                    <font color='#5555FF'>*</font>t <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>t;
                                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>cnt;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='col2img'></a>col2img</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> output,
            tensor<font color='#5555FF'>&amp;</font> data,
            <font color='#0000FF'><u>long</u></font> n,
            <font color='#0000FF'><u>long</u></font> filter_nr,
            <font color='#0000FF'><u>long</u></font> filter_nc,
            <font color='#0000FF'><u>long</u></font> stride_y,
            <font color='#0000FF'><u>long</u></font> stride_x,
            <font color='#0000FF'><u>long</u></font> padding_y,
            <font color='#0000FF'><u>long</u></font> padding_x
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> data.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
            <font color='#0000FF'>const</font> rectangle boundary <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> t <font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font><font color='#BB00BB'>output</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// now fill in the Toeplitz output matrix for the n-th sample in data.  
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_r <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_y<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nr<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_c <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_x<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nc<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_y; r <font color='#5555FF'>&lt;</font> max_r; r<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_y<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_x; c <font color='#5555FF'>&lt;</font> max_c; c<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_x<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> <font color='#979000'>0</font>; y <font color='#5555FF'>&lt;</font> filter_nr; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> x <font color='#5555FF'>=</font> <font color='#979000'>0</font>; x <font color='#5555FF'>&lt;</font> filter_nc; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>x<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'><u>long</u></font> xx <font color='#5555FF'>=</font> c<font color='#5555FF'>+</font>x;
                                <font color='#0000FF'><u>long</u></font> yy <font color='#5555FF'>=</font> r<font color='#5555FF'>+</font>y;
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>boundary.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font>xx,yy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                                    d[<font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> yy<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> xx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>t;
                                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>t;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> tensor_conv::<b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            resizable_tensor<font color='#5555FF'>&amp;</font> output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> data,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> filters,
            <font color='#0000FF'><u>int</u></font> stride_y,
            <font color='#0000FF'><u>int</u></font> stride_x,
            <font color='#0000FF'><u>int</u></font> padding_y,
            <font color='#0000FF'><u>int</u></font> padding_x
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>output,data<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>output,filters<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_y <font color='#5555FF'>&lt;</font> filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> padding_x <font color='#5555FF'>&lt;</font> filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y,
                "<font color='#CC0000'>Filter windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x,
                "<font color='#CC0000'>Filter windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;

            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                            filters.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                            <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y,
                            <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x<font face='Lucida Console'>)</font>;

            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> temp;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>img2col</font><font face='Lucida Console'>(</font>temp, data, n, filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, stride_y, stride_x, padding_y, padding_x<font face='Lucida Console'>)</font>;
                output.<font color='#BB00BB'>set_sample</font><font face='Lucida Console'>(</font>n, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>temp<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            last_stride_y <font color='#5555FF'>=</font> stride_y;
            last_stride_x <font color='#5555FF'>=</font> stride_x;
            last_padding_y <font color='#5555FF'>=</font> padding_y;
            last_padding_x <font color='#5555FF'>=</font> padding_x;
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> tensor_conv::
        <b><a name='get_gradient_for_data'></a>get_gradient_for_data</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> filters,
            tensor<font color='#5555FF'>&amp;</font> data_gradient
        <font face='Lucida Console'>)</font>
        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> temp;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n,
                              gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                              gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                    

                temp <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>gi<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>col2img</font><font face='Lucida Console'>(</font>temp, data_gradient, n, filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, last_stride_y, last_stride_x, last_padding_y, last_padding_x<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
        <font color='#0000FF'><u>void</u></font> tensor_conv::
        <b><a name='get_gradient_for_filters'></a>get_gradient_for_filters</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> data,
            tensor<font color='#5555FF'>&amp;</font> filters_gradient
        <font face='Lucida Console'>)</font>
        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> temp;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n,
                              gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                              gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


                <font color='#BB00BB'>img2col</font><font face='Lucida Console'>(</font>temp, data, n, filters_gradient.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filters_gradient.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, last_stride_y, last_stride_x, last_padding_y, last_padding_x<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    filters_gradient <font color='#5555FF'>=</font> gi<font color='#5555FF'>*</font>temp;
                <font color='#0000FF'>else</font>
                    filters_gradient <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi<font color='#5555FF'>*</font>temp;
            <b>}</b>
        <b>}</b>
    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#0000FF'><u>void</u></font> <b><a name='copy_tensor'></a>copy_tensor</b><font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> dest,
            <font color='#0000FF'><u>size_t</u></font> dest_k_offset,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> src,
            <font color='#0000FF'><u>size_t</u></font> src_k_offset,
            <font color='#0000FF'><u>size_t</u></font> count_k
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> dest_sample_size <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> src_sample_size <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> block_size <font color='#5555FF'>=</font> count_k <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                     dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>All sources should fit into dest tensor size</font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> dest_k_offset <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> count_k, "<font color='#CC0000'>Not enough space in dest tensor</font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src_k_offset <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> count_k, "<font color='#CC0000'>Not enough space in src tensor</font>"<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> dest_p <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> dest_k_offset <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> src_p <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src_k_offset <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            ::<font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>dest_p, src_p, block_size <font color='#5555FF'>*</font> <font color='#0000FF'>sizeof</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            dest_p <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dest_sample_size;
            src_p  <font color='#5555FF'>+</font><font color='#5555FF'>=</font> src_sample_size;
        <b>}</b>
    <b>}</b>

    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>    <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
    <b>}</b> 
<b>}</b>


<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_DNN_CPU_cPP_
</font>


</pre></body></html>